# 多线程取句子主干
import jieba.posseg
import re
from concurrent.futures import ThreadPoolExecutor
import threading

# 多线程用paddle各种莫名其妙的错
# jieba.enable_paddle()  # 开启paddle模式

keep = ["n", "f", "s", "nr", "ns", "nt", "nw", "nz",
        "v", "vd", "vn", "t", "m", "q", "TIME", "ORG", "LOC", "PER", "w"]
set_not = ["不", "非", "有", "无", "莫", "高度"]

# lock = threading.Lock()  # 申请一把锁

lines = []


def keep_main_content(sentence, fout, i):
    # print("i:",i)
    if i % 100 == 0:
        print("执行到了：%s 行" % i)
    if bool(re.search("[-\d()*+/=\.]{15,}", sentence)):
        return
    words = jieba.posseg.cut(sentence, use_paddle=False)
    source = []
    target = []
    for item in words:
        target.append(item.word)
        if item.flag in keep or item.word in set_not:
            # out = out + item.word + " "
            source.append(item.word)
    source = " ".join(source)
    target = " ".join(target)
    lines.append(source + "\t" + target + "\n")
    # lock.acquire()  # 加锁
    # fout.write('%s\n' % (source + "\t" + target))
    # lock.release()  # 释放锁


def handle(in1, out1):
    fp = open(in1, 'r', encoding='utf-8')
    # 'a'是在原先的文本的基础上进行直接增加
    # 'w'会覆盖之前的文本
    fout = open(out1, 'a', encoding='utf-8')
    fool = ThreadPoolExecutor(2)  # 线程池里有20个线程
    i = 0
    for line in fp.readlines():
        set_l = line.strip().split("\t")
        i += 1
        fool.submit(keep_main_content, set_l[1], fout, i)
    fool.shutdown()  # 默认为True,关闭了池的入口，会等待所有的任务执行完，结束阻塞
    print(len(lines))
    fout.writelines(lines)


if __name__ == '__main__':
    infile = 'g:/1.txt'
    outfile = 'g:/keep_content_bottom.tsv'
    handle(infile, outfile)
